{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "gpuType": "A100"
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "code",
      "source": [
        "!pip install codebleu"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "AMgGpkafsZJ9",
        "outputId": "1b12b936-b57d-4c0f-f53b-f4cb8cde8783"
      },
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting codebleu\n",
            "  Downloading codebleu-0.7.0-py3-none-any.whl.metadata (8.1 kB)\n",
            "Collecting tree-sitter<0.23.0,>=0.22.0 (from codebleu)\n",
            "  Downloading tree_sitter-0.22.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (10 kB)\n",
            "Requirement already satisfied: setuptools>=61.0.0 in /usr/local/lib/python3.11/dist-packages (from codebleu) (75.1.0)\n",
            "Downloading codebleu-0.7.0-py3-none-any.whl (31 kB)\n",
            "Downloading tree_sitter-0.22.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (544 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m544.2/544.2 kB\u001b[0m \u001b[31m12.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hInstalling collected packages: tree-sitter, codebleu\n",
            "Successfully installed codebleu-0.7.0 tree-sitter-0.22.3\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install javalang"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "YU9TNAgqtH2r",
        "outputId": "ded284bf-eec0-411d-fe83-8482ba6ca42c"
      },
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting javalang\n",
            "  Downloading javalang-0.13.0-py3-none-any.whl.metadata (805 bytes)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.11/dist-packages (from javalang) (1.17.0)\n",
            "Downloading javalang-0.13.0-py3-none-any.whl (22 kB)\n",
            "Installing collected packages: javalang\n",
            "Successfully installed javalang-0.13.0\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import javalang  # 需要安装javalang包: pip install javalang\n",
        "\n",
        "def parse_java(code):\n",
        "    try:\n",
        "        return javalang.parse.parse(code)\n",
        "    except:\n",
        "        return None"
      ],
      "metadata": {
        "id": "UG9vzp-ysW-8"
      },
      "execution_count": 15,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def calculate_weighted_bleu_java(ref, cand):\n",
        "    java_keywords = {\n",
        "        'abstract', 'assert', 'boolean', 'break', 'byte', 'case', 'catch',\n",
        "        'char', 'class', 'const', 'continue', 'default', 'do', 'double',\n",
        "        'else', 'enum', 'extends', 'final', 'finally', 'float', 'for',\n",
        "        'goto', 'if', 'implements', 'import', 'instanceof', 'int', 'interface',\n",
        "        'long', 'native', 'new', 'package', 'private', 'protected', 'public',\n",
        "        'return', 'short', 'static', 'strictfp', 'super', 'switch',\n",
        "        'synchronized', 'this', 'throw', 'throws', 'transient', 'try',\n",
        "        'void', 'volatile', 'while'\n",
        "    }\n",
        "\n",
        "    java_operators = {'=', '>', '<', '!', '~', '?', ':', '==', '<=', '>=', '!=',\n",
        "                     '&&', '||', '++', '--', '+', '-', '*', '/', '&', '|', '^',\n",
        "                     '%', '<<', '>>', '>>>', '+=', '-=', '*=', '/=', '&=', '|=',\n",
        "                     '^=', '%=', '<<=', '>>=', '>>>='}\n",
        "\n",
        "    ref_tokens = ref.split()\n",
        "    cand_tokens = cand.split()\n",
        "\n",
        "    weighted_matches = 0\n",
        "    for token in cand_tokens:\n",
        "        if token in ref_tokens:\n",
        "            if token in java_keywords:\n",
        "                weight = 3.0\n",
        "            elif token in java_operators:\n",
        "                weight = 2.5\n",
        "            else:\n",
        "                weight = 1.0\n",
        "            weighted_matches += weight\n",
        "\n",
        "    total_weights = sum(3.0 if t in java_keywords else\n",
        "                       2.5 if t in java_operators else\n",
        "                       1.0 for t in cand_tokens)\n",
        "\n",
        "    weighted_precision = weighted_matches / total_weights if total_weights > 0 else 0\n",
        "    brevity_penalty = min(1, len(cand_tokens) / len(ref_tokens)) if len(ref_tokens) > 0 else 0\n",
        "    return weighted_precision * brevity_penalty"
      ],
      "metadata": {
        "id": "k74eTgdwvV17"
      },
      "execution_count": 17,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def calculate_ast_similarity_java(ref, cand):\n",
        "    try:\n",
        "        ref_ast = parse_java(ref)\n",
        "        cand_ast = parse_java(cand)\n",
        "\n",
        "        if ref_ast is None or cand_ast is None:\n",
        "            return 0.0\n",
        "\n",
        "        # 简化的AST比较 - 实际实现可以更精细\n",
        "        ref_structure = str([type(node).__name__ for path, node in ref_ast])\n",
        "        cand_structure = str([type(node).__name__ for path, node in cand_ast])\n",
        "\n",
        "        if ref_structure == cand_structure:\n",
        "            return 1.0\n",
        "        else:\n",
        "            # 计算结构相似度\n",
        "            ref_nodes = set(type(node).__name__ for path, node in ref_ast)\n",
        "            cand_nodes = set(type(node).__name__ for path, node in cand_ast)\n",
        "            intersection = ref_nodes & cand_nodes\n",
        "            union = ref_nodes | cand_nodes\n",
        "            return len(intersection) / len(union) if union else 0.0\n",
        "    except:\n",
        "        return 0.0"
      ],
      "metadata": {
        "id": "WqKUybnzvWiy"
      },
      "execution_count": 18,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def calculate_data_flow_java(ref, cand):\n",
        "    try:\n",
        "        ref_ast = parse_java(ref)\n",
        "        cand_ast = parse_java(cand)\n",
        "\n",
        "        if ref_ast is None or cand_ast is None:\n",
        "            return 0.0\n",
        "\n",
        "        # 提取变量和方法信息\n",
        "        def extract_symbols(tree):\n",
        "            variables = set()\n",
        "            methods = set()\n",
        "\n",
        "            for path, node in tree:\n",
        "                if isinstance(node, javalang.tree.VariableDeclarator):\n",
        "                    variables.add(node.name)\n",
        "                elif isinstance(node, javalang.tree.MethodDeclaration):\n",
        "                    methods.add(node.name)\n",
        "                    # 添加参数\n",
        "                    for param in node.parameters:\n",
        "                        variables.add(param.name)\n",
        "\n",
        "            return variables, methods\n",
        "\n",
        "        ref_vars, ref_methods = extract_symbols(ref_ast)\n",
        "        cand_vars, cand_methods = extract_symbols(cand_ast)\n",
        "\n",
        "        # 计算相似度\n",
        "        var_intersection = ref_vars & cand_vars\n",
        "        method_intersection = ref_methods & cand_methods\n",
        "\n",
        "        var_sim = len(var_intersection) / len(ref_vars | cand_vars) if (ref_vars or cand_vars) else 1.0\n",
        "        method_sim = len(method_intersection) / len(ref_methods | cand_methods) if (ref_methods or cand_methods) else 1.0\n",
        "\n",
        "        return 0.7 * var_sim + 0.3 * method_sim  # 给变量相似度更高权重\n",
        "    except:\n",
        "        return 0.0"
      ],
      "metadata": {
        "id": "g68hc4jfvYm9"
      },
      "execution_count": 19,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def codebleu_java(reference_code, candidate_code, alpha=[0.25, 0.25, 0.25, 0.25]):\n",
        "    \"\"\"\n",
        "    Java专用的CodeBLEU评估函数\n",
        "    \"\"\"\n",
        "    # 1. 传统BLEU\n",
        "    bleu_score = calculate_bleu(reference_code, candidate_code)\n",
        "\n",
        "    # 2. Java加权BLEU\n",
        "    weighted_bleu = calculate_weighted_bleu_java(reference_code, candidate_code)\n",
        "\n",
        "    # 3. Java AST相似度\n",
        "    ast_similarity = calculate_ast_similarity_java(reference_code, candidate_code)\n",
        "\n",
        "    # 4. Java数据流相似度\n",
        "    data_flow_similarity = calculate_data_flow_java(reference_code, candidate_code)\n",
        "\n",
        "    # 组合分数\n",
        "    codebleu_score = (alpha[0] * bleu_score +\n",
        "                     alpha[1] * weighted_bleu +\n",
        "                     alpha[2] * ast_similarity +\n",
        "                     alpha[3] * data_flow_similarity)\n",
        "\n",
        "    return {\n",
        "        'codebleu': codebleu_score,\n",
        "        'bleu': bleu_score,\n",
        "        'weighted_bleu': weighted_bleu,\n",
        "        'ast_similarity': ast_similarity,\n",
        "        'data_flow_similarity': data_flow_similarity\n",
        "    }"
      ],
      "metadata": {
        "id": "LsONj66wvcGs"
      },
      "execution_count": 20,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import json\n",
        "with open('/content/qwenCoder_eval.json', 'r', encoding='utf-8') as file:\n",
        "    data = json.load(file)\n",
        "\n",
        "print(data[0][\"answer\"])\n",
        "answer1 = [item[\"answer\"] for item in data]"
      ],
      "metadata": {
        "id": "cHsARNG2vm5R"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "with open('/content/myModel_eval.json', 'r', encoding='utf-8') as file:\n",
        "    data = json.load(file)\n",
        "\n",
        "print(data[0][\"answer\"])\n",
        "answer2 = [item[\"answer\"] for item in data]"
      ],
      "metadata": {
        "id": "D-tqGEIhxwni"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "score_phi = []\n",
        "score_model = []\n",
        "avg_phi = 0\n",
        "avg_model = 0"
      ],
      "metadata": {
        "id": "Pc8To2FeyeoZ"
      },
      "execution_count": 57,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "i = 0\n",
        "for ans1, ans2 in zip(answer1, answer2):\n",
        "  print(f'question {i}')\n",
        "  i += 1\n",
        "  ref_java = ans1\n",
        "  cand_java = ans2\n",
        "  result = codebleu_java(ref_java, cand_java)\n",
        "  avg_model += result['codebleu']\n",
        "  # print(f\"CodeBLEU分数: {result['codebleu']:.4f}\")\n",
        "  # print(f\"详细分数: BLEU={result['bleu']:.4f}, 加权BLEU={result['weighted_bleu']:.4f}\")\n",
        "  # print(f\"AST相似度={result['ast_similarity']:.4f}, 数据流相似度={result['data_flow_similarity']:.4f}\")\n",
        "  score_model.append(result)\n",
        "print(f'avg_phi: {avg_model / 110}')"
      ],
      "metadata": {
        "collapsed": true,
        "id": "fSwKNyV9ym-3"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Java代码示例\n",
        "ref_java = \"\"\"\n",
        "public class HelloWorld {\n",
        "    public static void main(String[] args) {\n",
        "        System.out.println(\"Hello, World!\");\n",
        "    }\n",
        "}\n",
        "要判断一个数是否为素数，可以使用以下Java代码：\\n\\n```java\\npublic class PrimeChecker {\\n\\n    public static boolean isPrime(int number) {\\n        if (number <= 1) {\\n            return false;\\n        }\\n        if (number == 2) {\\n            return true;\\n        }\\n        if (number % 2 == 0) {\\n            return false;\\n        }\\n        \\n        int boundary = (int) Math.sqrt(number);\\n        \\n        for (int i = 3; i <= boundary; i += 2) {\\n            if (number % i == 0) {\\n                return false;\\n            }\\n        }\\n        \\n        return true;\\n    }\\n\\n    public static void main(String[] args) {\\n        int[] testNumbers = {2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20};\\n        \\n        for (int number : testNumbers) {\\n            System.out.println(\\\"Is \\\" + number + \\\" a prime number? \\\" + isPrime(number));\\n        }\\n    }\\n}\\n```\\n\\n这个程序定义了一个`isPrime`方法来检查一个数是否为素数。它首先处理特殊情况：小于等于1的数和2是素数，因为2是唯一能被1或2整除的数。对于大于2的数，它通过检查所有小于其平方根的数来排除非素数。\\n\\n在`main`方法中，我们测试了几个数，并打印出每个数是否为素数。\"\n",
        "    },\n",
        "\"\"\"\n",
        "\n",
        "cand_java = \"\"\"\n",
        "java\\npublic class PrimeChecker {\\n\\n    public static boolean isPrime(int number) {\\n        if (number <= 1) {\\n            return false;\\n        }\\n        if (number == 2) {\\n            return true;\\n        }\\n        if (number % 2 == 0) {\\n            return false;\\n        }\\n        \\n        int boundary = (int) Math.sqrt(number);\\n        \\n        for (int i = 3; i <= boundary; i += 2) {\\n            if (number % i == 0) {\\n                return false;\\n            }\\n        }\\n        \\n        return true;\\n    }\\n\\n    public static void main(String[] args) {\\n        int[] testNumbers = {2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20};\\n        \\n        for (int number : testNumbers) {\\n            System.out.println(\\\"Is \\\" + number + \\\" a prime number? \\\" + isPrime(number));\\n        }\\n    }\\n}\\n```\\n\\n这个程序定义了一个`isPrime`方法来检查一个数是否为素数。它首先处理特殊情况：小于等于1的数和2是素数，因为2是唯一能被1或2整除的数。对于大于2的数，它通过检查所有小于其平方根的数来排除非素数。\\n\\n在`main`方法中，我们测试了几个数，并打印出每个数是否为素数。\"\n",
        "    },\n",
        "\"\"\"\n",
        "\n",
        "result = codebleu_java(ref_java, cand_java)\n",
        "print(f\"CodeBLEU分数: {result['codebleu']:.4f}\")\n",
        "print(f\"详细分数: BLEU={result['bleu']:.4f}, 加权BLEU={result['weighted_bleu']:.4f}\")\n",
        "print(f\"AST相似度={result['ast_similarity']:.4f}, 数据流相似度={result['data_flow_similarity']:.4f}\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "-WRbgxdssX-A",
        "outputId": "e61583f3-db01-45cc-9668-4a1ed6a1db67"
      },
      "execution_count": 23,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "CodeBLEU分数: 0.4418\n",
            "详细分数: BLEU=0.8824, 加权BLEU=0.8849\n",
            "AST相似度=0.0000, 数据流相似度=0.0000\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "with open(\"score_model.json\", \"w\", encoding=\"utf-8\") as f:\n",
        "    json.dump(score_model, f, ensure_ascii=False, indent=4)  # indent=4 让 JSON 更易读"
      ],
      "metadata": {
        "id": "EPyMET6rvwq-"
      },
      "execution_count": 59,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "lzBnuKs9ztQT"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}